#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/10/25 12:48
# @Author  : supuchun
# @Site    : 
# @File    : n2_cc.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/10/23 09:08
# @Author  : supuchun
# @Site    :
# @File    : n2_create.py

from n2 import HnswIndex
import time
import numpy as np
import os
import redis
import pymysql

allfile = []


def getallfile(path):
    allfilelist = os.listdir(path)
    for file in allfilelist:
        filepath = os.path.join(path, file)
        allfile.append(filepath)
    return allfile


def getFilename(filename):
    (filepath, tempfilename) = os.path.split(filename);
    (shotname, extension) = os.path.splitext(tempfilename);
    return shotname


def getFileExtension(filename):
    (filepath, tempfilename) = os.path.split(filename);
    (shotname, extension) = os.path.splitext(tempfilename);
    return extension


def copy_to_csv_ready():
    cmd = "cp /data1/auto_add/*.csv /data1/auto_add/csv_ready/"
    # print(cmd)
    os.system(cmd)


def delete_from_csv_ready():
    cmd = "rm -f /data1/auto_add/csv_ready/*"
    os.system(cmd)


def get_true_filename(filename):
    tmplist = filename.split('_')
    # print(tmplist)
    # exit()
    sort = int(tmplist[1])
    asset_family = int(tmplist[2])
    csvno = str(tmplist[3])

    if (sort == 1 or sort == 2) and asset_family == 2:
        n2_prefix_name = 'vcgCreativeIndex'
    elif (sort == 1 or sort == 2) and asset_family == 1:
        n2_prefix_name = 'editvcg'
    elif sort == 3:
        n2_prefix_name = 'veerlog'
    elif sort == 4:
        n2_prefix_name = '500pxlog'
    elif sort == 5:
        n2_prefix_name = 'worldcup2018'
        tmp_af = 5
    elif sort == 6:
        n2_prefix_name = 'worldcupALL'
        tmp_af = 6

    true_csv_name = n2_prefix_name + csvno + 'M.n2'
    return true_csv_name


def read_successlog(filename, ):
    tmplist = filename.split('_')
    sort = tmplist[1]
    asset_family = tmplist[2]
    csvno = tmplist[3]


def open_db():
    global db, cursor
    db = pymysql.connect("rm-2zezq40w18v8dqcy2.mysql.rds.aliyuncs.com", "root", "2bWP*qU8W$%", "new_add")
    cursor = db.cursor()


def close_db():
    global db
    db.close()


# 写入消息队列，参数1：队列名称，参数2：消息内容
def update_ele_index(index_no, index_id, company_id,spider_id):
    global db, cursor
    tbname = 'new_add.ele_index_info'
    tmp_af = 8

    # print(type(asset_family))
    # exit()
    sql = """update %s set index_no=%d, index_id=%d, asset_family=%d where company_id='%s' AND spider_id ='%s'""" % (
        tbname, index_no, index_id, tmp_af, company_id,spider_id)
    # print(sql)
    # exit()
    try:
        # 执行sql语句
        cursor.execute(sql)
        # 提交到数据库执行
        db.commit()
    except:
        # 如果发生错误则回滚
        print(sql)
        db.rollback()
        return False

    return True

print('start:' + str(time.time()))

# # 第一步：将每日追加的csv临时文件，拷贝到csv_ready目录：#########################################
#copy_to_csv_ready()
#print('step 1 done:' + str(time.time()))
#
# 第二步：开始跑csv_ready目录下的csv文件######################################################
myPath = '/home/supuchun/n2_demo/'
allfiles = getallfile(myPath)
for item in allfiles:
    f = 2048  # Weight
    t = HnswIndex(f)  # HnswIndex(f, "L2 or angular")
    filename = 'feature_01_demo'
    sucesslogName = 'sucesslog' + filename
    errorlogName = 'errorlog' + filename
    with open(item, 'r') as fi, open(errorlogName, 'w') as fo, open(sucesslogName, 'w') as fo1:
        for i in range(6912):
            chunk_data = fi.readline()
            if not chunk_data:
                print('chunk_data is done')
                break
            myID = chunk_data.split('\t')[0]
            strVector = chunk_data.split('\t')[1]
            features_arr = strVector.split(',')
            features_arr = [a for a in features_arr if len(a) > 0]
            features_arr = np.array(features_arr, dtype=np.float64)

            try:
                t.add_data(features_arr)
                successInfo = myID + '\n'
                fo1.write(successInfo)
            except:
                errInfo = myID + '|' + strVector + '\n'
                fo.write(errInfo)

    t.build(m=1000, n_threads=32)
    taget_index_name = filename + '.n2'
    t.save(taget_index_name)
print('step 2 done:' + str(time.time()))
# exit()

# 第三步：循环全部successlog文件 写入redis######################################################
'''open_db()
myPath = '/home/supuchun/'
allsuccessfiles = getallfile(myPath)
r = redis.Redis(host='r-2zea3c0ff18d2e94.redis.rds.aliyuncs.com', port=6379, db=0, password='b3bPVU8W',
                decode_responses=True)

for item in allsuccessfiles:
    filename = "sucesslogfeature_01_demo"

    if 'sucesslog' in filename:
        #tmplist = filename.split('_')
        #csvno = int(tmplist[3])
        fd = open('/home/supuchun/' + filename)
        # print('/data1/auto_add/n2_ready/'+filename)
        # exit()
        index_no = 1
        index_id = 0
        for line in fd:
            if index_id % 1000 == 0:
                print(index_id)
            result_id = line.strip()
            taskInfo = result_id.split('-')
            company_id = taskInfo[0]
            spider_id = taskInfo[1]
            r_key = '%d_%d_%d' % (index_no, index_id, 8)
            if r.exists(r_key) != True:
                tr = r.set(r_key, result_id)
		print tr
                update_ele_index(index_no, index_id, company_id,spider_id)
            index_id += 1

close_db()
print('step 3 done:' + str(time.time()))'''


# 第四步：将n2临时文件名 改为 在线文件名################################################
'''myPath = '/data1/auto_add/n2_ready/'
allcsvfiles = getallfile(myPath)

for item in allcsvfiles:
    ext = str(getFileExtension(item))
    filename = str(getFilename(item))

    if ext == '.n2':
        true_n2_name = get_true_filename(filename)
        cmd = "mv %s /data1/auto_add/n2_ready/%s" % (item, true_n2_name)
        print(cmd)
        os.system(cmd)

print('step 4 done:' + str(time.time()))'''